import scrapy
from lxml import etree
import json
from pymongo import MongoClient
from ..items import TutorialItem
import  random


class QuotesSpider(scrapy.Spider):
    name ="qcr"


    def start_requests(self):
        # scrapy.log.start()
        urls =[
        'http://www.shicimingju.com/shicimark',
        ]
        for url in urls:
            # scrapy.log.msg(url, level=INFO, spider=None)
            yield scrapy.Request(url=url, callback=self.parse)
    def parse(self, response):
        item=response.css('.layui-col-sm7 div div')
        for i in  item:
            href=i.css('div a::attr(href)').extract_first()
            title=i.css('div a::text').extract_first()

            if href is not None :
                newHref = 'http://www.shicimingju.com' + str(href)
                yield scrapy.Request(url=newHref, callback=self.getDetailList)

    def  getDetailList(self,response):
        item = response.css('.layui-col-sm7 div')

        pagination = response.css('.pagination a::attr(href)').getall()

        if pagination is not None:
            plen = (len(pagination)) - 1
            newHref = 'http://www.shicimingju.com' + str(pagination[plen])
            print(newHref)
            yield scrapy.Request(url=newHref, callback=self.getDetailList)

        for i in item:
            href = i.css('h3 a::attr(href)').extract_first()

            # title = i.css('div a::text').extract_first()
            if href is not None:
                newHref = 'http://www.shicimingju.com' + str(href)
                yield scrapy.Request(url=newHref, callback=self.getDetail)

    def getDetail(self,response):
        title = response.css('.shici-title::text').extract_first()
        niandai = response.css('.shici-info::text').extract_first().strip()
        shiren = response.css('.shici-info a::text').extract_first().strip()
        content =  (response.css('.shici-content::text').getall())
        newContent=""
        item = TutorialItem()

        for cj in content :
            newContent+=cj

        item['_id'] = title + str(random.randint(1, 1000))
        item["title"]=title
        item["niandai"]=niandai
        item["shiren"]=shiren
        item["content"]=newContent
        print(item)
        yield item





